import math
import torch
from torch import nn
from d2l import torch as d2l

#@save
def transpose_qkv(X:torch.Tensor, num_heads):
    print(f'before reshape: {X.shape}')
    # 拆分维度，讲维度2拆分为num_heads和他俩的商，张量的元素数量不变
    X = X.reshape(X.shape[0], X.shape[1], num_heads, -1)
    print(f'after  reshape: {X.shape}')
    # 更换维度顺序，将维度1和2更换顺序
    X = X.permute(0, 2, 1, 3)
    print(f'after  permute: {X.shape}')
    print(f'return : {X.reshape(-1, X.shape[2], X.shape[3]).shape}')
    # 合并0， 1两个维度
    return X.reshape(-1, X.shape[2], X.shape[3])
#@save
def transpose_output(X:torch.Tensor, num_heads):

    X = X.reshape(-1, num_heads, X.shape[1], X.shape[2])
    X = X.permute(0, 2, 1, 3)
    return X.reshape(X.shape[0], X.shape[1], -1)

#@save
class MultiHeadAttention(nn.Module):
    '''多头注意力'''
    def __init__(self, key_size, query_size, value_size, num_hiddens,
                num_heads, dropout, bias=False, **kwargs):
        super(MultiHeadAttention, self).__init__(**kwargs)
        self.num_heads = num_heads
        # 点注意力 不用学w
        self.attention = d2l.DotProductAttention(dropout)
        self.W_q = nn.Linear(query_size, num_hiddens, bias=bias)
        self.W_k = nn.Linear(key_size, num_hiddens, bias=bias)
        self.W_v = nn.Linear(value_size, num_hiddens, bias=bias)
        self.W_o = nn.Linear(num_hiddens, num_hiddens, bias=bias)

    def forward(self, queries, keys, values, valid_lens):

        queries = transpose_qkv(self.W_q(queries), self.num_heads)
        keys = transpose_qkv(self.W_k(keys), self.num_heads)
        values = transpose_qkv(self.W_v(values), self.num_heads)
        
        if valid_lens is not None:
            # 向量重复，参数分别是：重复源，重复次数，重复所在维度
            valid_lens = torch.repeat_interleave(
                valid_lens, repeats=self.num_heads, dim=0
            )

        output = self.attention(queries, keys, values, valid_lens)

        output_concat = transpose_output(output, self.num_heads)
        return self.W_o(output_concat)

num_hiddens, num_heads = 100, 5
attention = MultiHeadAttention(num_hiddens, num_hiddens, num_hiddens, num_hiddens, num_heads, 0.5)
attention.eval()

batch_size = 2
num_queries = 4
num_kvpairs = 6
valid_lens = torch.tensor([3,2])
X = torch.ones((batch_size, num_queries, num_hiddens))
Y = torch.ones((batch_size, num_kvpairs, num_hiddens))
print(attention(X, Y, Y, valid_lens).shape)